{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "1ab224be", "metadata": { "scrolled": true }, "outputs": [], "source": [ "import os\n", "import requests\n", "from datetime import datetime, timedelta\n", "\n", "!pip install aiohttp\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "d277ed5c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: nest_asyncio in c:\\users\\magda\\anaconda3\\lib\\site-packages (1.6.0)\n" ] } ], "source": [ "!pip install nest_asyncio\n", "\n", "import aiohttp\n", "import asyncio\n", "from bs4 import BeautifulSoup\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "dc3405a1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: requests in c:\\users\\magda\\anaconda3\\lib\\site-packages (2.31.0)\n", "Requirement already satisfied: beautifulsoup4 in c:\\users\\magda\\anaconda3\\lib\\site-packages (4.12.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\magda\\anaconda3\\lib\\site-packages (from requests) (2.0.4)\n", "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\magda\\anaconda3\\lib\\site-packages (from requests) (3.4)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\magda\\anaconda3\\lib\\site-packages (from requests) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\magda\\anaconda3\\lib\\site-packages (from requests) (2024.2.2)\n", "Requirement already satisfied: soupsieve>1.2 in c:\\users\\magda\\anaconda3\\lib\\site-packages (from beautifulsoup4) (2.5)\n" ] } ], "source": [ "!pip install requests beautifulsoup4\n" ] }, { "cell_type": "code", "execution_count": 12, "id": "72c2b99b", "metadata": { "scrolled": true }, "outputs": [], "source": [ "import os\n", "import requests\n", "from bs4 import BeautifulSoup\n", "from datetime import datetime, timedelta\n", "from concurrent.futures import ThreadPoolExecutor, as_completed\n", "\n", "def download_file(url, file_path):\n", " \"\"\"\n", " Download a file from a URL and save it to a specified path.\n", " \n", " :param url: URL to download\n", " :param file_path: Local path to save the downloaded file\n", " \"\"\"\n", " try:\n", " response = requests.get(url, stream=True)\n", " response.raise_for_status() # Raise an error for bad status\n", " with open(file_path, 'wb') as file:\n", " for chunk in response.iter_content(chunk_size=8192):\n", " if chunk:\n", " file.write(chunk)\n", " print(f\"Downloaded: {file_path}\")\n", " except requests.exceptions.RequestException as e:\n", " print(f\"Failed to download {url}: {e}\")\n", "\n", "def download_nc_files_from_directory(base_url, day_dir):\n", " \"\"\"\n", " Download all .nc files from a given directory using parallel downloads.\n", " \n", " :param base_url: Base URL for the directory containing .nc files\n", " :param day_dir: Local directory to save the downloaded files\n", " \"\"\"\n", " try:\n", " response = requests.get(base_url)\n", " response.raise_for_status()\n", " soup = BeautifulSoup(response.text, 'html.parser')\n", " \n", " # Gather all .nc file URLs\n", " urls = []\n", " for link in soup.find_all('a'):\n", " href = link.get('href')\n", " if href.endswith('LV1.nc'): #LV1.nc for CR\n", " file_url = f\"{base_url}/{href}\"\n", " file_path = os.path.join(day_dir, href)\n", " urls.append((file_url, file_path))\n", " \n", " # Download files in parallel\n", " with ThreadPoolExecutor(max_workers=10) as executor:\n", " future_to_url = {executor.submit(download_file, url, path): url for url, path in urls}\n", " for future in as_completed(future_to_url):\n", " url = future_to_url[future]\n", " try:\n", " future.result()\n", " except Exception as e:\n", " print(f\"Error downloading {url}: {e}\")\n", " \n", " except requests.exceptions.RequestException as e:\n", " print(f\"Failed to access {base_url}: {e}\")\n", "\n", "def download_files_for_month(base_url_template, start_date, end_date, base_dir):\n", " \"\"\"\n", " Download all .nc files for every day in a given month.\n", " \n", " :param base_url_template: Template for the base URL with placeholders for date and hour\n", " :param start_date: Start date of the month (format: 'YYYY-MM-DD')\n", " :param end_date: End date of the month (format: 'YYYY-MM-DD')\n", " :param base_dir: The base directory where files should be stored\n", " \"\"\"\n", " current_date = datetime.strptime(start_date, '%Y-%m-%d')\n", " end_date_obj = datetime.strptime(end_date, '%Y-%m-%d')\n", "\n", " while current_date <= end_date_obj:\n", " date_str = current_date.strftime('%Y-%m-%d')\n", " year_month = current_date.strftime('%Y-%m')\n", " day_str = current_date.strftime('%d')\n", " base_url = f\"https://ruisdael.citg.tudelft.nl/reform/Cloud_Radar/Data/Y{current_date.strftime('%Y')}/M{current_date.strftime('%m')}/D{day_str}\"\n", " day_dir = os.path.join(base_dir, year_month, date_str)\n", "\n", " if not os.path.exists(day_dir):\n", " os.makedirs(day_dir)\n", "\n", " print(f\"Downloading files for {date_str}...\")\n", " download_nc_files_from_directory(base_url, day_dir)\n", "\n", " current_date += timedelta(days=1)\n", "\n", "# Base URL and directory settings\n", "#base_url_template = 'https://ruisdael.citg.tudelft.nl/reform/Cloud_Radar/Data/Y{year}/M{month}/D{day}'\n", "#base_dir = 'C:\\\\Users\\\\magda\\\\Master_Thesis\\\\Cloud_radar'\n", "#start_date = '2024-03-21'\n", "#end_date = '2024-03-31'\n", "\n", "# Run the download process\n", "#download_files_for_month(base_url_template, start_date, end_date, base_dir)\n" ] }, { "cell_type": "code", "execution_count": 18, "id": "851207fa", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Downloading Sonic Anemometer files for 2024-07-01...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_120000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_210000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-01\\20240701_220000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-02...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_120000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_220000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-02\\20240702_210000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-03...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_120000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_210000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-03\\20240703_220000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-04...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_120000_sonic.dat\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_220000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-04\\20240704_210000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-05...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_120000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_220000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-05\\20240705_210000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-06...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_120000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_220000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-06\\20240706_210000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-07...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_120000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_210000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-07\\20240707_220000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-08...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_060000_sonic.dat\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_120000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_210000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-08\\20240708_220000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-09...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_090000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_080000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_020000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_100000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_110000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_130000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_160000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_120000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_230000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_150000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_140000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_170000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_180000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_190000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_210000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_200000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-09\\20240709_220000_sonic.dat\n", "Downloading Sonic Anemometer files for 2024-07-10...\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-10\\20240710_040000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-10\\20240710_060000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-10\\20240710_030000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-10\\20240710_050000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-10\\20240710_070000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-10\\20240710_010000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-10\\20240710_000000_sonic.dat\n", "Downloaded: C:\\Users\\magda\\Master_Thesis\\Sonic\\2024-07\\2024-07-10\\20240710_020000_sonic.dat\n" ] } ], "source": [ "def download_files_from_directory(base_url, day_dir, file_extension):\n", " \"\"\"\n", " Download all files with a given extension from a directory using parallel downloads.\n", " \n", " :param base_url: Base URL for the directory containing the files\n", " :param day_dir: Local directory to save the downloaded files\n", " :param file_extension: File extension to filter for downloading (e.g., '.nc', '.dat')\n", " \"\"\"\n", " try:\n", " response = requests.get(base_url)\n", " response.raise_for_status()\n", " soup = BeautifulSoup(response.text, 'html.parser')\n", " \n", " # Gather all file URLs with the given extension\n", " urls = []\n", " for link in soup.find_all('a'):\n", " href = link.get('href')\n", " if href.endswith(file_extension):\n", " file_url = f\"{base_url}/{href}\"\n", " file_path = os.path.join(day_dir, href)\n", " urls.append((file_url, file_path))\n", " \n", " # Download files in parallel\n", " with ThreadPoolExecutor(max_workers=10) as executor:\n", " future_to_url = {executor.submit(download_file, url, path): url for url, path in urls}\n", " for future in as_completed(future_to_url):\n", " url = future_to_url[future]\n", " try:\n", " future.result()\n", " except Exception as e:\n", " print(f\"Error downloading {url}: {e}\")\n", " \n", " except requests.exceptions.RequestException as e:\n", " print(f\"Failed to access {base_url}: {e}\")\n", "\n", "\n", "\n", "def download_sonic_anemometer_files_for_month(base_url_template, start_date, end_date, base_dir):\n", " \"\"\"\n", " Download all Sonic Anemometer files with '.dat' extension for every day in a given month.\n", " \n", " :param base_url_template: Template for the base URL with placeholders for date\n", " :param start_date: Start date of the month (format: 'YYYY-MM-DD')\n", " :param end_date: End date of the month (format: 'YYYY-MM-DD')\n", " :param base_dir: The base directory where files should be stored\n", " \"\"\"\n", " current_date = datetime.strptime(start_date, '%Y-%m-%d')\n", " end_date_obj = datetime.strptime(end_date, '%Y-%m-%d')\n", "\n", " while current_date <= end_date_obj:\n", " date_str = current_date.strftime('%Y-%m-%d')\n", " year_month = current_date.strftime('%Y-%m')\n", " day_str = current_date.strftime('%d')\n", " month_str = current_date.strftime('%m')\n", " base_url = base_url_template.format(year=current_date.strftime('%Y'), month=month_str, day=day_str)\n", " day_dir = os.path.join(base_dir, year_month, date_str)\n", "\n", " if not os.path.exists(day_dir):\n", " os.makedirs(day_dir)\n", "\n", " print(f\"Downloading Sonic Anemometer files for {date_str}...\")\n", " download_files_from_directory(base_url, day_dir, '.dat')\n", "\n", " current_date += timedelta(days=1)\n", "\n", "# Settings for Sonic Anemometer\n", "base_url_template_sonic = 'https://ruisdael.citg.tudelft.nl/reform/Sonic_Anemometer/Y{year}/M{month}/D{day}'\n", "base_dir_sonic = 'C:\\\\Users\\\\magda\\\\Master_Thesis\\\\Sonic'\n", "start_date_sonic = '2024-07-01'\n", "end_date_sonic = '2024-07-10'\n", "\n", "# Run the download process for Sonic Anemometer\n", "download_sonic_anemometer_files_for_month(base_url_template_sonic, start_date_sonic, end_date_sonic, base_dir_sonic)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b014d5d0", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.7" } }, "nbformat": 4, "nbformat_minor": 5 }